package br.edu.ufcg.ccc.util;

import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.StringTokenizer;

import org.apache.commons.lang3.StringUtils;

/**
 * 
 * @author victorclf
 * @author pedroyossis
 */
public class TwitterParseUtil {
	private static final String HTML_TAG_START = "<a";
	private static final String HTML_TAG_END = "</a>";
	
	public static String preformatLine(String token) {
		token = token.toLowerCase();
		token = removeHTMLTags(token);
		token = removeSpecialCharTags(token);
		token = removeWeirdCharacters(token);
		token = trimToken(token);
		token = removeExtraSpaces(token);
		return token;
	}
	
	public static String preformatToken(String token) {
		token = trimToken(token);
		token = removeExtraSpaces(token);
		return token;
	}
	
	public static boolean isTrivialWord(String token) {
		return (token.length() <= 2 && StringUtils.isAlpha(token))
				|| Constants.getTrivialWordSet().contains(token);
	}

	public static boolean isValidWord(String token) {
		return token != null && !token.trim().isEmpty();
	}
	
	public static List<String> tokenize(String line) {
		StringTokenizer tokenizer = new StringTokenizer(line);
		List<String> tokens = new LinkedList<String>();
		
		while (tokenizer.hasMoreTokens()) {
			tokens.add(tokenizer.nextToken());
		}
		
		return tokens;
	}
	
	public static List<String> getValidTokens(List<String> rawTokens) {
		List<String> validTokens = new ArrayList<String>();
		
		for (String token : rawTokens) {
			token = preformatToken(token);
			
			if (TwitterParseUtil.isValidWord(token) && !TwitterParseUtil.isTrivialWord(token)) {
				validTokens.add(token);
			}
		}
		
		return validTokens;
	}

	private static String removeHTMLTags(String line) {
		while (true) {
			int startPos = line.indexOf(HTML_TAG_START);
			if (startPos < 0) break;
				
			int endPos = line.indexOf(HTML_TAG_END, startPos + 1);
			
			if (endPos < 0) endPos = line.length();
			else endPos += HTML_TAG_END.length();

			line = line.substring(0, startPos) + line.substring(endPos);
		}
		
		return line;
	}
	
	private static String removeSpecialCharTags(String line) {
		return line.replaceAll("&[a-z0-9;]*;", "");
	}

	private static String trimToken(String token) {
		return StringUtils.strip(token, " -");
	}

	private static String removeExtraSpaces(String token) {
		return token.replaceAll(" {2,}", " ");
	}

	private static String removeWeirdCharacters(String token) {
		return token.replaceAll("[^\\p{L}0-9\\- ]", " ");
	}
}
